package extractorCode;


import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import java.io.*;

public class boingboing {
	
	private final static int INITIAL_BUF_SIZE = 65536;

	public static void main(String args[]) throws Exception
	{
		
		int count=0;
		BufferedReader br = new BufferedReader(new FileReader("c:\\newData\\Boing\\boing_urls.txt"));
		
		String url;
	
		while((url=br.readLine())!=null)
		{	
		System.out.println(url);
			
		try
		{
		String page = (fetchPage(url));
		//System.out.println(url+"&singlepage=true");
	//	System.out.println(page);
		page= page.replaceAll(">\\s*<", "><");
		
		page = page.replaceAll("\t", "");
		page = page.replaceAll("\b", "");
		page = page.replaceAll("\r", "");
		page = page.replaceAll("\n", "");
		page = page.replaceAll("\r\n", "");
		
	//	System.out.println(page);
		Matcher newsItem = Pattern.compile("</h2><p class=\"byline\">.*?</p>(.*?)<div class=\"shareMe\">", Pattern.CASE_INSENSITIVE).matcher( page );
		
		
		
		
		if(newsItem.find())
		{
			System.out.println(count);
		//System.out.println(newsItem.group(1));
		String text = newsItem.group(1);
		//System.out.println(text);
		text = text.replaceAll("<p class=\"byline\">.*?</p>", "");
		text = text.replaceAll("\\<.*?>","");
		text = text.replaceAll("[^\\p{ASCII}]", "");
		System.out.println(text);
		
		BufferedWriter wr = new BufferedWriter( new FileWriter("C:\\newData\\Boing\\"+(count++)+".txt"));
		wr.write(text);
		wr.close();
		
	//	System.out.println(text);
	//	writer.write("http://newsbusters.org"+newsItem.group(1)+"\n");
		}
		else
		{
			
			System.out.println(url);
		}
		
		}
		catch(Exception e)
		{
			
			
			
		}
		
		
		}
		
		br.close();
		

	}
	
	private static String fetchPage(String urlString){
		StringBuilder buffer = new StringBuilder(INITIAL_BUF_SIZE);
		try {
			URL url = new URL( urlString );
HttpURLConnection conn = (HttpURLConnection)url.openConnection();
            
            conn.addRequestProperty("user-agent","Mozilla/5.0");
            BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
			int temp = reader.read();
			while(temp != -1){
				buffer.append((char)temp);
				temp = reader.read();
			}
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return buffer.toString();
	}

}
